import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Hashtable;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class testMain {
	/**
	 * @param args
	 */
	private static String picPathBase = "/home/mengchang/CrawlDianping/storePic";
	private static long sleepMs = 500;
	private BufferedWriter bw;
	private BufferedWriter bwResult;
	private String currentId;
	private String currentName;
	private String currentAddress;
	
	
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		testMain tmpobj = new testMain();
		tmpobj.doReadAndWrite();
	}
	
	private int countFileLine(String filePath) throws IOException {
		int fileLineCount = 0;
		FileReader fr = new FileReader(filePath);
	
		BufferedReader br = new BufferedReader(fr);
		int fileLine = 0;
		while(br.readLine()!=null){
			fileLine++;
		}
		System.out.println(filePath+"====="+fileLine);
		
		fr.close();
		br.close();
		
		return fileLine;
	}
	
	private void doReadAndWrite() {
		try {
			// read file content from file
			// StringBuffer sb= new StringBuffer("");
			
			String sourceFilename = "/home/mengchang/CrawlDianping/storeList.csv";
			String logFileName =	"/home/mengchang/CrawlDianping/storeInfo.log";
			String resultFileName = "/home/mengchang/CrawlDianping/storeInfo.csv";
			
			int fileLineNum = this.countFileLine(sourceFilename);
			
			FileReader reader = new FileReader(sourceFilename);
			BufferedReader br = new BufferedReader(reader);

			FileWriter writer = new FileWriter(logFileName);
			this.bw = new BufferedWriter(writer);

			FileWriter writerResult = new FileWriter(resultFileName);
			this.bwResult = new BufferedWriter(writerResult);
			ArrayList<String> failList = new ArrayList<String>();
			
			String str = null;
			int loopCount = 0;
			while ((str = br.readLine()) != null) {
				++ loopCount;
				this.bw.write("\n\n"+loopCount+" of "+ fileLineNum + "\nstring read : "+str+"\n");
				System.out.println("\n\n"+loopCount+" of "+ fileLineNum + "\nstring read : "+str);
				if(this.singleLineLogic(str)){
					
				} else {
					failList.add(str);
				}
				//break;
			}
			
			loopCount = 0;
			int totalcount = failList.size();
			for(String tmpstr : failList) {
				++ loopCount;
				this.bw.write("\n\n failList "+loopCount+" of "+ totalcount + "\nstring read : "+tmpstr+"\n");
				System.out.println("\n\n"+loopCount+" of "+ totalcount + "\nstring read : "+tmpstr);
				this.singleLineLogic(tmpstr);
			}

			br.close();
			reader.close();

			this.bwResult .close();
			writerResult.close();
			
			this.bw.close();
			writer.close();
			
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	private boolean singleLineLogic(String str) {
		try {
			String[] tmpStrList = str.split(",");
			if (tmpStrList.length > 4) {
				Thread.sleep(testMain.sleepMs);
				if(this.findStore(tmpStrList[3], tmpStrList[1], tmpStrList[4], tmpStrList[0])) {
					this.bw.flush();
					this.bwResult.flush();
					return true;
				} 
				return false;
			}
			// String strURL1 = "http://www.dianping.com/shop/2545675";
			// tmpobj.captureDianpingHtml(strURL1);
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			return false;
		}
		return false;
	}
	
	private boolean downloadPic(String picurlStr, String picStorePath) {
		URL picurl = null;
		try {
			picurl = new URL(picurlStr);
		} catch (MalformedURLException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
	
		InputStream is = null;
		try {
			is = picurl.openStream();
		} catch (IOException e) {
			e.printStackTrace();
			return false;
		}
	
		OutputStream os = null;
		try{
			os = new FileOutputStream(picStorePath);
			int bytesRead = 0;
			byte[] buffer = new byte[8192];
			while((bytesRead = is.read(buffer,0,8192))!=-1){
				os.write(buffer,0,bytesRead);
			}
			
			os.close();
		}catch(FileNotFoundException e){
			e.printStackTrace();
			return false;
		} catch (IOException e) {
			e.printStackTrace();
			return false;
		}
		
		try {
			is.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		return true;
	}

	private boolean findStore(String title, String city, String address, String id) {
		String strURL = "http://www.baidu.com/s?wd=";
		strURL += "site:dianping.com";
		strURL += "+" + title;
		strURL += "+" + city;
		strURL += "+" + address;
		
		this.currentId = id;
		this.currentName = title;
		this.currentAddress = address;
		
		boolean findStoreSucc = false;
		try {
			findStoreSucc = this.captureBaiduHtml(strURL);
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			findStoreSucc = false;
		}
		
		return findStoreSucc;
	}

	public boolean captureBaiduHtml(String strURL) throws Exception {
		System.out.println("in func captureBaiduHtml : " + strURL);
		this.bw.write("in func captureBaiduHtml : " + strURL+"\n");
		URL url = new URL(strURL);
		HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
		InputStreamReader input = new InputStreamReader(
				httpConn.getInputStream(), "utf-8");
		BufferedReader bufReader = new BufferedReader(input);
		String line = "";
		StringBuilder contentBuf = new StringBuilder();
		while ((line = bufReader.readLine()) != null) {
			contentBuf.append(line);
		}
		
		Hashtable<String, String> urlVisitTable = new Hashtable<String, String>();
		
		
		String buf = contentBuf.toString();
		boolean isFindValidInfo = false;
		int loopcount = 0;
		int fromIndex = 0;
		while (true) {
			++loopcount;
			if (loopcount > 20) {
				break;
			}
			int firstIndex = buf.indexOf("http://www.baidu.com/link?url=", fromIndex);
			if (firstIndex > 0) {
				int endIndex = buf.indexOf("\"", firstIndex);
				if (endIndex > 0) {
					String targetUrl = buf.substring(firstIndex, endIndex);
					fromIndex = endIndex;
					if(null == urlVisitTable.get(targetUrl)){
						System.out.println(targetUrl);
						urlVisitTable.put(targetUrl, "");
						if (this.captureDianpingHtml(targetUrl)) {
							isFindValidInfo = true;
							break;
						}
						
					} else {
						System.out.println("skip duplicate: "+targetUrl);
					}
				} else {
					break;
				}
			} else {
				break;
			}
		}// end of while

		return isFindValidInfo;
	}

	private String getRealUrl(String urlstr) throws IOException {
		URL url = new URL(urlstr);

		HttpURLConnection conn=(HttpURLConnection)url.openConnection();
		conn.getResponseCode();
		String realUrl=conn.getURL().toString();
		conn.disconnect();
		
		return realUrl;
	}
	
	public boolean captureDianpingHtml(String strURL) throws Exception {
		System.out.println("in func captureDianpingHtml : " + strURL);
		this.bw.write("in func captureDianpingHtml : " + strURL+"\n");
		
		String realUrl = this.getRealUrl(strURL);
		if(realUrl.length() ==0 ){
			return false;
		}
		
		// realurl should like http://www.dianping.com/shop/5741527
		System.out.println("in func captureDianpingHtml realurl: " + realUrl);
		if( 0 != realUrl.indexOf("http://www.dianping.com/shop/")) {
			System.out.println("skip wrong : " + realUrl);
			this.bw.write("skip wrong : " + realUrl+"\n");
			return false;
		}
		
		// http://www.dianping.com/shop/4025434/map
		if(realUrl.indexOf("/map")>=0) {
			System.out.println("skip wrong : " + realUrl);
			this.bw.write("skip wrong : " + realUrl+"\n");
			return false;
		}
		
		URL url = new URL(realUrl);
		HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
		httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7");

		InputStreamReader input = new InputStreamReader(
				httpConn.getInputStream(), "utf-8");
		BufferedReader bufReader = new BufferedReader(input);
		String line = "";
		StringBuilder contentBuf = new StringBuilder();
		while ((line = bufReader.readLine()) != null) {
			contentBuf.append(line);
		}

		String buf = contentBuf.toString();
		//System.out.println(buf);

		Thread.sleep(testMain.sleepMs);
		return this.parseDianpingHtml(buf, realUrl);
	}
	
	private String findFromHtml(String html, String key, String start, String end,
			int fromindex) {
		if (key.length() == 0 || start.length() == 0 || end.length() == 0) {
			return "";
		}
		if (fromindex < 0) {
			fromindex = 0;
		}
		int keyIndex = html.indexOf(key, fromindex);
		if (keyIndex < 0) {
			return "";
		}
		int startIndex = html.indexOf(start, keyIndex);
		if (startIndex < 0) {
			return "";
		}
		int endIndex = html.indexOf(end, startIndex + start.length());
		if (endIndex < 0) {
			return "";
		}
		return html.substring(startIndex + start.length(), endIndex);
	}

	private static final String regEx_script = "<script[^>]*?>[\\s\\S]*?<\\/script>"; // 定义script的正则表达式
    private static final String regEx_style = "<style[^>]*?>[\\s\\S]*?<\\/style>"; // 定义style的正则表达式
    private static final String regEx_html = "<[^>]+>"; // 定义HTML标签的正则表达式
    
    private static String delHTMLTag(String htmlStr) {
        Pattern p_script = Pattern.compile(regEx_script, Pattern.CASE_INSENSITIVE);
        Matcher m_script = p_script.matcher(htmlStr);
        htmlStr = m_script.replaceAll(""); // 过滤script标签

        Pattern p_style = Pattern.compile(regEx_style, Pattern.CASE_INSENSITIVE);
        Matcher m_style = p_style.matcher(htmlStr);
        htmlStr = m_style.replaceAll(""); // 过滤style标签

        Pattern p_html = Pattern.compile(regEx_html, Pattern.CASE_INSENSITIVE);
        Matcher m_html = p_html.matcher(htmlStr);
        htmlStr = m_html.replaceAll(""); // 过滤html标签

        return htmlStr.trim(); // 返回文本字符串
    }
    
	private boolean parseDianpingHtml(String htmlstr, String urlstr) {
		// <h1 class="shop-title" itemprop="name itemreviewed">威尔士健身会所(绿洲中环店)</h1>
		String titleStr = this.findFromHtml(htmlstr,
				"<h1 class=\"shop-title\"", ">", "</h1", 0);
		System.out.println(titleStr);
		int subtitleIndex = titleStr.indexOf("(");
		String subtitle = "";
		if (subtitleIndex > 0) {
			subtitle = titleStr.substring(subtitleIndex);
			subtitle = subtitle.replace("(", " ");
			subtitle = subtitle.replace(")", " ");
			subtitle = subtitle.trim();
			System.out.println(subtitle);
		}
		// <em itemprop="count">67</em>
		String countStr = this.findFromHtml(htmlstr, "<em itemprop=\"count\"",
				">", "</em", 0);
		System.out.println(countStr);
		float countfloat = 0.0f;
		if (countStr.length() > 0) {
			countfloat = Float.parseFloat(countStr);
			countfloat /= 10.0f;
			System.out.println(countfloat);
		}
		// <img title="威尔士健身会所的图片" itemprop="photo" alt="威尔士健身会所的图片"  src="http://i3.dpfile.com/pc/8c948f18bb619a48d4880e81b211e6ff/14810750_m.jpg"/>
		String imgStr = this.findFromHtml(htmlstr, "itemprop=\"photo\"",
				"src=\"", "\"", 0);
		System.out.println(imgStr);
		String picDownloadPath = "";
		String picName = "";
		if(imgStr.length() > 0){
			int lastindex = imgStr.lastIndexOf(".");
			if(lastindex > 0){
				String suffix = imgStr.substring(lastindex+1);
				picDownloadPath = testMain.picPathBase+"/"+this.currentId+"."+suffix;
				picName = this.currentId+"."+suffix;
				if(this.downloadPic(imgStr, picDownloadPath)) {
					System.out.println("succ to download pic and store : "+ picDownloadPath);
					try {
						this.bw.write("succ to download pic and store : "+ picDownloadPath + "\n");
					} catch (IOException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}
				}
			}
		}

		// <span itemprop="street-address">金沙江路1688号7号楼3楼(近真北路)</span>
		String addrStr = this.findFromHtml(htmlstr,
				"itemprop=\"street-address\"", ">", "</span", 0);
		System.out.println(addrStr);

		// <div
		// class="J_brief-cont">从停车场上来很绕，第一次找了很久也没找到怎么上去。....</div>
		String commentStr = this.findFromHtml(htmlstr,
				"class=\"J_brief-cont\"", ">", "</div", 0);
		commentStr = testMain.delHTMLTag(commentStr);
		commentStr = commentStr.trim();
		commentStr = commentStr.replaceAll("\n", "");
		commentStr = commentStr.replaceAll(",", "，");
		System.out.println(commentStr);

		if (titleStr.length() == 0 || addrStr.length() == 0) {
			return false;
		}

		try {
			this.bw.write("titleStr:" + titleStr + "\n");
			this.bw.write("subtitle:" + subtitle + "\n");
			this.bw.write("countfloat:" + countfloat + "\n");
			this.bw.write("imgStr:" + imgStr + "\n");
			this.bw.write("addrStr:" + addrStr + "\n");
			this.bw.write("commentStr:" + commentStr + "\n");
			
			this.bwResult.write(this.currentId+","+this.currentName+","+this.currentAddress+","+urlstr+","+titleStr+","+subtitle+","+countfloat+","+imgStr+","+picName+","+addrStr+","+commentStr+"\n");

		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		return true;
	}
}
